package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"
)

func main() {
	startUrl := "http://www.wangshuge.la/books/74/74550/12827324.html"
	crawl3(startUrl, "gotest.txt")
}

// 爬取小说内容并写入text文件
func crawl3(startUrl string, fileName string) {
	c := colly.NewCollector(
		colly.AllowedDomains("www.ibiquges.com", "www.xbiquge.la", "www.xbiquges.la", "www.wangshuge.la"),
		//colly.AllowURLRevisit(),
	)

	// 使用随机user-agent
	extensions.RandomUserAgent(c)

	// HTTP 的配置
	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment, // 使用代理
		DialContext: (&net.Dialer{
			Timeout:   40 * time.Second, // 超时时间
			KeepAlive: 40 * time.Second, // keepAlive 超时时间
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,              // 最大空闲连接数
		IdleConnTimeout:       90 * time.Second, // 空闲连接超时
		TLSHandshakeTimeout:   10 * time.Second, // TLS 握手超时
		ExpectContinueTimeout: 5 * time.Second,
	})

	// 设置请求信息
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("Host", "www.xbiquge.la")
		r.Headers.Set("X-Requested-With", "XMLHttpRequest")
		r.Headers.Set("Referer", "https://www.xbiquge.la/13/13959/24775996.html")
		r.Headers.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0")

		log.Println("Visiting", r.URL)
	})

	c.OnResponse(func(r *colly.Response) {
		log.Println("response received", r.StatusCode)
	})

	// 解析章节内容 并 写入文本文件
	c.OnHTML("dd[id='contents']", func(e *colly.HTMLElement) {
		// title
		title := e.DOM.ParentsUntil("body").Find("#amain dl dd h1").Text() + "\n\n"
		title = strings.Trim(title, " ")
		left := strings.Index(title, ".")
		right := strings.Index(title, " ")
		title = title[0:left] + title[right:] + "\n"
		println(title)

		// content
		e.DOM.Find("p").Remove()
		content := e.DOM.Text()
		str := title + content

		writeToText3(str, fileName)
	})

	// 翻页
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		e.DOM.EachWithBreak(func(i int, s *goquery.Selection) bool {
			if "下一页" == s.Text() {
				baseUrl, _ := s.Attr("href")
				url := "http://www.wangshuge.la/" + baseUrl

				c.Visit(url)
				return false
			}
			return true
		})
	})

	c.OnError(func(r *colly.Response, err error) {
		log.Println("error:", r.StatusCode, err)
	})

	c.Visit(startUrl)

}

// 将字符串，写入文本文件
func writeToText3(str string, fileName string) {
	filePath := "D:\\crawl\\" + fileName
	//wd, _ := os.Getwd()
	//filePath := wd + "/src/fiction-master/file/" + fileName        // 存放小说的TXT文件路径
	f, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) // 以追加方式写入
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err = f.Close(); err != nil {
			log.Fatal(err)
		}
	}()
	_, err = fmt.Fprintln(f, str)
	if err != nil {
		log.Fatal(err)
	}
}
